# -*- coding: utf-8 -*- 
# @Time : 2022/12/1 22:43 
# @Author : 拼搏的小浣熊 
# @File : robotparser.py
from urllib.request import urlopen
from urllib.robotparser import RobotFileParser

rp = RobotFileParser()
# rp.set_url("https://www.baidu.com/robots.txt")

rp.parse(urlopen("https://www.baidu.com/robots.txt").read().decode('utf-8').split('\n'))
print(rp.can_fetch( 'Baiduspider', 'https://www.baidu.com' ))
print(rp.can_fetch( 'Baiduspider ', 'https://www.baidu.com/homepage/ '))
print(rp.can_fetch( ' Googlebot', 'https:/ /www.baidu.com/homepagel/'))
